import requests
from data import dict_data
from bs4 import BeautifulSoup

# 书籍地址
url = "https://fanqienovel.com/page/6982529841564224526"

# 网站域名
host = "https://fanqienovel.com"

# 获取网页的源代码
html = requests.get(url).text

bs = BeautifulSoup(html, "lxml")
chapters = bs.select_one("div.page-directory-content div div.chapter")

# 获取所有的章节
chapter_items = chapters.select("div.chapter-item a")

for item in chapter_items:
    title, href = item.text, item["href"]
    chapter_url = host + href  # 章节url

    print(f"-----------正在采集 {title}---------------")
    file_dir = "D:/projects/spider/day06字体反爬2/凡尘神域/" + title + ".txt"
    # 为每一章节创建一个文件
    fp = open(file_dir, "a", encoding="utf-8")
    html2 = requests.get(chapter_url).text

    bs2 = BeautifulSoup(html2, "lxml")
    p_list = bs2.select("div.muye-reader-content  p")

    for p in p_list:
        text = p.text
        new_text = ""
        for char in text:
            try:
                new_char = dict_data[str(ord(char))]
            except KeyError:
                new_char = char
            new_text += new_char

        fp.write(new_text + "\n")

    print(f"------------------{title} 采集完毕！！！")






